{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d3e1e95c-61c0-40ec-960c-40f6b8b3f714",
   "metadata": {},
   "source": [
    "## 如何实现拒答\n",
    "\n",
    "1. 把所有该拒答和不该拒答的问题收集起来（没有数据的时候，mock一部分问题）\n",
    "2. 计算这些问题与知识库里面的相似分数分布\n",
    "   1. 倒排方面，查看倒排得分逻辑，优化不合理得分\n",
    "   2. 向量方面，如果与正反馈的分数分布区分明显，直接找到该阈值，否则跳到下一步\n",
    "3. 拿上面两种进行向量微调，再次查看分数分布区分度，如果不合格再跳转到下一步\n",
    "4. 拿上面两种进行Rerank，再次查看分数分布区分度\n",
    "5. 如果区分度还存在问题，低风险分段进行提示词次软性拒答，高风险分段进行硬性拒答"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3df42c5d-2c87-4cf6-ad77-cf0d1322f460",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install boto3\n",
    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git -Uq\n",
    "!pip install -Uq sentence-transformers \n",
    "!pip install faiss-cpu -Uq"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82374d33-dedf-4a23-870a-d71ec0eadca9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 1 整理全部应该拒答的问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "58f7cd49-93bc-437d-9d2a-5af724f7c272",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from boto3 import client as boto3_client\n",
    "import boto3\n",
    "import os\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "Tune_Specified_Chat_Name = 'OnlyForDEBUG'\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "embedding_endpoint_name = 'bge-zh-15-2023-09-17-01-00-27-086-endpoint'\n",
    "\n",
    "lambda_client = boto3_client('lambda')\n",
    "\n",
    "def generate_train_dataset(query, embedding_endpoint_name):\n",
    "    data = {\n",
    "        'OPENAI_API_KEY': '', \n",
    "        'ws_endpoint': 'https://7bk6ty9c65.execute-api.us-west-2.amazonaws.com/Prod', \n",
    "        'msgid': 'id-1695293211608-da6836af1c57_res', \n",
    "        'chat_name': Tune_Specified_Chat_Name, \n",
    "        'prompt': query, \n",
    "        'model': 'chatglm', \n",
    "        'use_qa': True, \n",
    "        'imgurl': None, \n",
    "        'template_id': 'default', \n",
    "        'max_tokens': 4000, \n",
    "        'temperature': 0.01, \n",
    "        'system_role': 'AWSBot', \n",
    "        'system_role_prompt': '你是亚马逊云科技的智能客服', \n",
    "        'embedding_model': embedding_endpoint_name,\n",
    "        'channel_cnt' : 100\n",
    "    }\n",
    "\n",
    "    invoke_response = lambda_client.invoke(FunctionName='Ask_Assistant', \n",
    "                                           InvocationType='RequestResponse',\n",
    "                                           Payload=json.dumps(data))\n",
    "    ret_obj = json.loads(invoke_response['Payload'].read().decode('utf-8'))\n",
    "    if 'body' not in ret_obj.keys():\n",
    "        print(ret_obj)\n",
    "    info = ret_obj['body'][0]['extra_info']\n",
    "    query = info['query_input']\n",
    "    bm25_infos = info['opensearch_query_response']\n",
    "    bm25_scores = [ item['score'] for item in bm25_infos ] \n",
    "    knn_infos = info['opensearch_knn_respose']\n",
    "\n",
    "    bm25_negs = []\n",
    "    knn_negs = []\n",
    "    knn_qq_negs = []\n",
    "    knn_qd_negs = []\n",
    "    knn_qq_scores = []\n",
    "    knn_qd_scores = []\n",
    "    for item in knn_infos:\n",
    "        knn_negs.append(item['doc'])\n",
    "        mid = item['doc'].split('Answer: ')\n",
    "        q = mid[0].split('Question: ')[1]\n",
    "        a = mid[1]\n",
    "        \n",
    "        if item['doc_type'] == 'Question':\n",
    "            knn_qq_negs.append(q)\n",
    "            knn_qq_scores.append(item['score'])\n",
    "        elif item['doc_type'] == 'Paragraph':\n",
    "            knn_qd_negs.append(a)\n",
    "            knn_qd_scores.append(item['score'])\n",
    "    \n",
    "    for item in bm25_infos:\n",
    "        bm25_negs.append(item['doc'])\n",
    "    \n",
    "    return query, bm25_negs, bm25_scores, knn_qq_negs, knn_qq_scores, knn_qd_negs, knn_qd_scores\n",
    "\n",
    "# query, bm25_negs, bm25_scores, knn_qq_negs, knn_qq_scores, knn_qd_negs, knn_qd_scores = generate_train_dataset(query, embedding_endpoint_name)\n",
    "\n",
    "def get_vector_by_sm_endpoint(questions, sm_client, endpoint_name, instruction=''):\n",
    "    response_model = sm_client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": questions,\n",
    "                \"is_query\": True,\n",
    "                \"instruction\" :  instruction,\n",
    "                \"parameters\" : \"\"\n",
    "            }\n",
    "        ),\n",
    "        ContentType=\"application/json\",\n",
    "    )\n",
    "    # 中文instruction => 为这个句子生成表示以用于检索相关文章：\n",
    "    # English instruction => Represent this sentence for searching relevant passages:\n",
    "    json_str = response_model['Body'].read().decode('utf8')\n",
    "    json_obj = json.loads(json_str)\n",
    "    embeddings = json_obj['sentence_embeddings']\n",
    "    return embeddings[0]\n",
    "\n",
    "def similarity_calc(vec1, vec2):\n",
    "    dot_product = np.dot(vec1, vec2)\n",
    "    norm_vec1 = np.sqrt(np.dot(vec1, vec1))\n",
    "    norm_vec2 = np.sqrt(np.dot(vec2, vec2))\n",
    "    cosine_sim = dot_product / (norm_vec1 * norm_vec2)\n",
    "    \n",
    "    return cosine_sim\n",
    "\n",
    "def get_similiarity(sent1, sent2):\n",
    "    vec1 = get_vector_by_sm_endpoint(sent1, smr_client, embedding_endpoint_name)\n",
    "    vec2 = get_vector_by_sm_endpoint(sent2, smr_client, embedding_endpoint_name)\n",
    "    return similarity_calc(vec1, vec2)\n",
    "\n",
    "def plot_stat(pos_similarity_list, neg_similarity_list):\n",
    "    \n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.set_style(\"whitegrid\")\n",
    "    L1 = len(pos_similarity_list)\n",
    "    L2 = len(neg_similarity_list)\n",
    "    if L1 > L2:\n",
    "        sample_neg_similarity_list = random.sample(neg_similarity_list, L2)\n",
    "        sample_pos_similarity_list = pos_similarity_list\n",
    "    else:\n",
    "        sample_neg_similarity_list = neg_similarity_list\n",
    "        sample_pos_similarity_list = random.sample(pos_similarity_list, L1)\n",
    "    \n",
    "    sns.histplot(data=sample_pos_similarity_list, bins=20)\n",
    "    sns.histplot(data=sample_neg_similarity_list, bins=20)\n",
    "    \n",
    "    # plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "46a15196-5e7c-4036-8ca2-01d374be8cba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 72/72 [00:44<00:00,  1.63it/s]\n"
     ]
    }
   ],
   "source": [
    "all_neg_bm25_scores = []\n",
    "all_neg_knn_qq_scores = []\n",
    "all_neg_knn_qd_scores = []\n",
    "\n",
    "neg_feedback_qa_triple = []\n",
    "neg_feedback_qq_triple = []\n",
    "\n",
    "# mock out_of_range question\n",
    "with open('aws_glue.faq') as mock_outofrange_faq:\n",
    "    all_content = mock_outofrange_faq.read()\n",
    "    qa_list = all_content.split('=====')\n",
    "    for qa in tqdm(qa_list):\n",
    "        parts = qa.split('\\nAnswer: ')\n",
    "        q = parts[0].split('Question: ')[1]\n",
    "        a = parts[1]\n",
    "        query, bm25_negs, bm25_scores, knn_qq_negs, knn_qq_scores, knn_qd_negs, knn_qd_scores = generate_train_dataset(q, embedding_endpoint_name)\n",
    "        all_neg_bm25_scores.extend(bm25_scores)\n",
    "        all_neg_knn_qq_scores.extend(knn_qq_scores)\n",
    "        all_neg_knn_qd_scores.extend(knn_qd_scores)\n",
    "        neg_feedback_qa_triple.append({\"query\":q, \"pos\": [a], \"neg\": bm25_negs + knn_qd_negs})\n",
    "        neg_feedback_qq_triple.append({\"query\":q, \"pos\": [], \"neg\": knn_qq_negs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fb135d5f-552a-4ede-b718-e4a3d850b8e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#mock questionable question\n",
    "all_pos_knn_qq_scores = []\n",
    "all_pos_knn_qd_scores = []\n",
    "\n",
    "with open('aws_emr_offical.json') as mock_normal_qlist:\n",
    "    lines = mock_normal_qlist.readlines()\n",
    "    for line in lines:\n",
    "        json_obj = json.loads(line)\n",
    "        o_query = json_obj['origin_question'].strip()\n",
    "        o_answer = json_obj['origin_answer'].strip()\n",
    "        g_query = json_obj['generate_question'].strip()\n",
    "        g_answer = json_obj['generate_answer'].strip()\n",
    "        \n",
    "        sim_qq = get_similiarity(g_query, o_query)\n",
    "        sim_qa1 = get_similiarity(g_query, o_answer)\n",
    "        sim_qa2 = get_similiarity(g_query, g_answer)\n",
    "        sim_qa3 = get_similiarity(o_query, o_answer)\n",
    "        all_pos_knn_qq_scores.append(sim_qq)\n",
    "        all_pos_knn_qd_scores.append(sim_qa1)\n",
    "        all_pos_knn_qd_scores.append(sim_qa2)\n",
    "        all_pos_knn_qd_scores.append(sim_qa3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e580edb-6134-4171-aee9-58e7edeae65f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(f\"len of all_pos_knn_qq_scores: {len(all_pos_knn_qq_scores)}\")\n",
    "print(all_pos_knn_qq_scores[:20])\n",
    "print(f\"len of all_neg_knn_qq_scores: {len(all_neg_knn_qq_scores)}\")\n",
    "print(all_neg_knn_qq_scores[:20])\n",
    "print(f\"len of all_pos_knn_qd_scores: {len(all_pos_knn_qd_scores)}\")\n",
    "print(all_pos_knn_qd_scores[:20])\n",
    "print(f\"len of all_neg_knn_qd_scores: {len(all_neg_knn_qd_scores)}\")\n",
    "print(all_neg_knn_qd_scores[:20])\n",
    "\n",
    "print(f\"len of neg_feedback_qq_triple: {len(neg_feedback_qq_triple)}\")\n",
    "print(neg_feedback_qq_triple[:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a8b4281-db2d-4a88-bdb3-a10249ae9c3d",
   "metadata": {},
   "source": [
    "### Step2 统计分数分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39ff4bc8-2d93-480b-b20d-3892b9c4b4b8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_stat(all_pos_knn_qq_scores, all_neg_knn_qq_scores)\n",
    "plot_stat(all_pos_knn_qd_scores, all_neg_knn_qd_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf83347f-8d2d-4d07-9c31-68469e041eb0",
   "metadata": {},
   "source": [
    "### Step3 构建Finetune 数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "891a9db9-fc5a-4a9b-95e3-06012e333db0",
   "metadata": {},
   "source": [
    "### 构造finetune数据集\n",
    "- 如果没有QQ数据集，通过LLM给query换个提问角度,得到 query_mock\n",
    "- 如果没有QD数据集，通过LLM给query给出一个回答,得到 answer_mock\n",
    "- 融合知识库里面现有的问答，得到供训练的所需正负例\n",
    "    + (instruction+query, answer_mock, all_retrival_neg_answers)\n",
    "    + (query, query_mock, all_retrival_neg_querys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1986ae86-ce9b-4e3f-b900-03972d24f777",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "llm_endpoint_name='bc2-7b-stream-2023-09-21-10-07-30-023-endpoint'\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "\n",
    "def mock_qq_dataset(llm_endpoint_name, smr_client, neg_feedback_qq_triple):\n",
    "    mock_query_template=\"\"\"给问题\"{}\"生成一个相似的问题\"\"\"\n",
    "    parameters = {\n",
    "      \"max_length\": 1024,\n",
    "      \"temperature\": 0.99,\n",
    "      \"top_p\":0.99\n",
    "    }\n",
    "    \n",
    "    def call_llm(prompt, smr_client):\n",
    "        response_model = smr_client.invoke_endpoint(\n",
    "                    EndpointName=llm_endpoint_name,\n",
    "                    Body=json.dumps(\n",
    "                    {\n",
    "                        \"inputs\": prompt,\n",
    "                        \"parameters\": parameters,\n",
    "                        \"history\" : []\n",
    "                    }\n",
    "                    ),\n",
    "                    ContentType=\"application/json\",\n",
    "                )\n",
    "\n",
    "        json_obj=json.loads(response_model['Body'].read().decode('utf8'))\n",
    "        return json_obj['outputs']\n",
    "    \n",
    "\n",
    "    for example_obj in tqdm(neg_feedback_qq_triple):\n",
    "        prompt = mock_query_template.format(example_obj['query'])\n",
    "        example_obj[\"pos\"] = [ call_llm(prompt, smr_client).strip('\"')]\n",
    "        \n",
    "    return neg_feedback_qq_triple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b03a86e2-5b9e-41a3-8b87-5a64db3d0663",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 72/72 [00:44<00:00,  1.61it/s]\n"
     ]
    }
   ],
   "source": [
    "neg_feedback_qq_triple = mock_qq_dataset(llm_endpoint_name, smr_client, neg_feedback_qq_triple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b3c5c6-b40c-4964-af9a-ed25e66b5a88",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "neg_feedback_qq_triple[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7ad954e9-a712-4f21-81e7-4c6f737f3f5e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cnt: 963\n"
     ]
    }
   ],
   "source": [
    "data_size = len(neg_feedback_qq_triple)\n",
    "qq_train_data_set = neg_feedback_qq_triple[:int(data_size*0.9)]\n",
    "qq_test_data_set =  neg_feedback_qq_triple[int(data_size*0.9):]\n",
    "\n",
    "if not os.path.exists('./trainset'):\n",
    "    os.mkdir('./trainset')\n",
    "if not os.path.exists('./testset'):\n",
    "    os.mkdir('./testset')\n",
    "\n",
    "with open('./trainset/refuse_trainset.jsonl', 'w') as file1:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in qq_train_data_set ]\n",
    "    file1.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./testset/refuse_testset.jsonl', 'w') as file2:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in qq_test_data_set ]\n",
    "    file2.write(\"\\n\".join(lines))\n",
    "    \n",
    "knowledgebase_qq_triple = []\n",
    "knowledgebase_gqoa_triple = []\n",
    "knowledgebase_gqga_triple = []\n",
    "all_g_q = []\n",
    "all_o_a = []\n",
    "all_g_a = []\n",
    "\n",
    "with open('aws_emr_offical.json') as mock_normal_qlist:\n",
    "    lines = mock_normal_qlist.readlines()\n",
    "    for idx, line in enumerate(lines):\n",
    "        json_obj = json.loads(line)\n",
    "        o_query = json_obj['origin_question'].strip()\n",
    "        o_answer = json_obj['origin_answer'].strip()\n",
    "        g_query = json_obj['generate_question'].strip()\n",
    "        g_answer = json_obj['generate_answer'].strip()\n",
    "        all_g_q.append(g_query)\n",
    "        all_o_a.append(o_answer)\n",
    "        all_g_a.append(g_answer)\n",
    "        knowledgebase_qq_triple.append({\"query\": o_query, \"pos\": [g_query], \"neg\" : idx})\n",
    "        knowledgebase_gqoa_triple.append({\"query\": g_query, \"pos\": [o_answer], \"neg\" : idx})\n",
    "        knowledgebase_gqga_triple.append({\"query\": g_query, \"pos\": [g_answer], \"neg\" : idx})\n",
    "        \n",
    "kg_data_size = len(knowledgebase_qq_triple)\n",
    "train_cnt = int(kg_data_size*0.9)\n",
    "print(f\"train_cnt: {train_cnt}\")\n",
    "\n",
    "for idx in range(kg_data_size):\n",
    "    knowledgebase_qq_triple[idx][\"neg\"] = all_g_q[:idx] + all_g_q[idx+1:]\n",
    "    knowledgebase_gqoa_triple[idx][\"neg\"] = all_o_a[:idx] + all_o_a[idx+1:]\n",
    "    knowledgebase_gqga_triple[idx][\"neg\"] = all_g_a[:idx] + all_g_a[idx+1:]\n",
    "    \n",
    "with open('./trainset/kg_qq_trainset.jsonl', 'w') as file3:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_qq_triple[:train_cnt] ]\n",
    "    file3.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./testset/kg_qq_testset.jsonl', 'w') as file4:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_qq_triple[train_cnt:] ]\n",
    "    file4.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./trainset/kg_gqoa_trainset.jsonl', 'w') as file5:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_gqoa_triple[:train_cnt] ]\n",
    "    file5.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./testset/kg_gqoa_testset.jsonl', 'w') as file6:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_gqoa_triple[train_cnt:] ]\n",
    "    file6.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./trainset/kg_gqga_trainset.jsonl', 'w') as file7:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_gqga_triple[:train_cnt] ]\n",
    "    file7.write(\"\\n\".join(lines))\n",
    "    \n",
    "with open('./testset/kg_gqga_testset.jsonl', 'w') as file8:\n",
    "    lines = [ json.dumps(example, ensure_ascii=False) for example in knowledgebase_gqga_triple[train_cnt:] ]\n",
    "    file8.write(\"\\n\".join(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "77f2a2ad-a99e-4d19-86be-873b13d50bce",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1070"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(knowledgebase_gqga_triple)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04235ad5-7967-4a84-972c-c979bdcf8f79",
   "metadata": {},
   "source": [
    "### step4 Finetune Rerank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3a2f0240-3af2-4f70-bf94-558c30499964",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cat ./trainset/*.jsonl > trainset/kg_merged.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8972d9a7-f253-4e70-b373-c48d49ba44a7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!wc -l trainset/kg_merged.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d40f2a1-236b-4f18-badd-105c67dd56c4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cat ./testset/kg_*.jsonl > ./testset/kg_merged.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e9bee9c-7c02-424f-965c-df83d3d89966",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!wc -l testset/kg_merged.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69026e49-33e2-41ed-88c5-e912e2e1dcd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!torchrun --nproc_per_node 1 \\\n",
    "-m FlagEmbedding.reranker.run \\\n",
    "--output_dir ./refuse_rerank_bge_base_zh15 \\\n",
    "--model_name_or_path BAAI/bge-reranker-base \\\n",
    "--train_data ./trainset \\\n",
    "--learning_rate 6e-5 \\\n",
    "--fp16 \\\n",
    "--num_train_epochs 3 \\\n",
    "--per_device_train_batch_size 1 \\\n",
    "--gradient_accumulation_steps 4 \\\n",
    "--dataloader_drop_last True \\\n",
    "--train_group_size 9 \\\n",
    "--max_len 512 \\\n",
    "--weight_decay 0.01 \\\n",
    "--logging_steps 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba5ff960-886f-4433-8a34-02e669eb369a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from FlagEmbedding import FlagReranker\n",
    "reranker = FlagReranker('./refuse_rerank_bge_base_zh15', use_fp16=True)\n",
    "\n",
    "scores = reranker.compute_score([['什么是 AWS Glue？', 'AWS Glue 是一项无服务器数据集成服务，它简化了发现、准备和合并数据以进行分析、机器学习和应用程序开发的工作。 AWS Glue 提供数据集成所需的全部功能，使您只需几分钟时间便可以开始分析数据并将数据投入使用，而不用耗时几个月。AWS Glue 为您提供可视化界面和基于代码的界面来简化数据集成。用户使用 AWS Glue 数据目录可以轻松找到并访问数据。数据工程师和 ETL（即提取、转换和加载）开发人员只需在 AWS Glue Studio 中点击几次，便能够以可视化的方式创建、运行和监控 ETL 工作流程。数据分析师和数据科学家可以使用 AWS Glue DataBrew 可视化方式丰富、清理和标准化数据，而无需编写代码。']])\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "92f379da-c233-4c19-b30e-6ab36ea700ec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compute Scores: 100%|██████████| 24/24 [00:08<00:00,  2.95it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsAAAAH5CAYAAACPsogXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6dElEQVR4nO3df3RU9Z3/8ddk8puEEEnARhCqJGnThCRSGgnJIrilraW1SKt0PeDSLktqi8WvQikalF8nGaFyutDaLGhaWgE9YHu20NMtLttTC25N96SaKMQU64+AQjJiSMhPJvf7BzBLCiGZ5M7MJZ/n4xwPzr2f+Xw+d96Zmdd8cufGZVmWJQAAAMAQEeGeAAAAABBKBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwSmS4J3Ct6Onp0blz5xQRESGXyxXu6QAAAODvWJalnp4eRUZGKiKi73VeAvAAnTt3TjU1NeGeBgAAAPqRk5Oj6OjoPvcTgAfo4qeInJwcud3uy/b7fD7V1NT0uR+hRT2ch5o4C/VwFurhLNTDWQKpx8W2V1v9lQjAA3bxtAe3233VB7+//Qgt6uE81MRZqIezUA9noR7OEkg9+jtdlS/BAQAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAADNHTYxkxZn8iwz0BAAAAhEZEhEtbDtbr+On2kIx3Q3Kcls5KD8lYgSAAAwAAGOT46Xa97T0b7mmEFadAAAAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKGENwFVVVSopKVFRUZEyMzP14osv9tl29erVyszM1E9/+tNe27u6urRu3ToVFBQoLy9PJSUl+uCDD3q1aW5u1vLlyzVlyhRNmTJFy5cv15kzZ4JxSAAAAHC4sAbgtrY2ZWZmavXq1Vdt9+KLL+rVV1/VmDFjLtu3YcMGHThwQJs3b9bOnTvV1tamJUuWyOfz+ds89NBDOnr0qLZv367t27fr6NGjWrFihe3HAwAAAOeLDOfgM2bM0IwZM67a5uTJk1q7dq2efvppLVmypNe+lpYW7d27V0888YQKCwslSRs3btRtt92mw4cPq7i4WMeOHdNLL72k559/Xrm5uZKkdevW6Z577tFbb72lm266KaA5Xxqsr7S9r/0ILerhPNTEWaiHs1APZxnO9XC73ZIkywrtuEN5LAOpx0DHCWsA7k9PT4+WL1+ub37zm0pPT79sf21trbq7uzV9+nT/trFjxyo9PV3V1dUqLi5WdXW1EhMT/eFXkvLy8pSYmKjq6uqAA3BNTc2Q9iO0qIfzUBNnoR7OQj2cZbjVIy4uTllZWWpra1Nra2tIxmyLd0mS6urq1N7ePqS+7KyHowPwtm3bFBkZqYULF15xf1NTk6KiopSUlNRre0pKipqamvxtRo8efdl9R48e7W8TiJycHP+np0v5fD7V1NT0uR+hRT2ch5o4C/VwFurhLMO9HvHx8UpICM0ScHx8vCQpMzNz0H0EUo+Lbfvj2ABcW1urHTt26IUXXpDL5QrovtYA1vUtywq4X+n8rw6u9uD3tx+hRT2ch5o4C/VwFurhLMO5HoOIQENix+NoZz0cexm0P//5z/J6vZo5c6aysrKUlZWl48ePy+PxaNasWZLOr/R2d3erubm51329Xq9SUlL8bbxe72X9f/jhh1dcGQYAAMDw5tgV4DvvvNP/xbaLvvnNb+rOO+/UXXfdJUnKzs5WVFSUDh06pDvuuEOSdOrUKdXX12v58uWSpPz8fLW0tOi1117T5MmTJUmvvvqqWlpalJ+fH8IjAgAAgBOENQCfPXtW7777rv92Q0ODjhw5oqSkJKWlpSk5OblX+6ioKKWkpPi/uJaYmKh58+bJ4/EoOTlZSUlJ8ng8ysjI8Ifnm2++WcXFxXr00Ue1du1aSVJpaalmzpwZ8BfgAAAAcO0LawCura3t9QW3srIySdLcuXNVXl4+oD5WrVqlyMhILVu2TB0dHZo2bZrKy8t7nSOyadMmrV+/Xt/4xjckSbNmzer32sMAAAAYnsIagAsKClRXVzfg9gcPHrxsW0xMjEpLS1VaWtrn/UaNGqVNmzYNao4AAAAYXhz7JTgAAAAgGAjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUcIagKuqqlRSUqKioiJlZmbqxRdf9O/r7u7Wxo0b9aUvfUl5eXkqKirSihUrdPLkyV59dHV1ad26dSooKFBeXp5KSkr0wQcf9GrT3Nys5cuXa8qUKZoyZYqWL1+uM2fOhOQYAQAA4CxhDcBtbW3KzMzU6tWrL9vX0dGhN954Q9/61rf0wgsvaOvWrXr77bf1rW99q1e7DRs26MCBA9q8ebN27typtrY2LVmyRD6fz9/moYce0tGjR7V9+3Zt375dR48e1YoVK4J+fAAAAHCeyHAOPmPGDM2YMeOK+xITE1VZWdlr26OPPqqvfe1rOnHihNLS0tTS0qK9e/fqiSeeUGFhoSRp48aNuu2223T48GEVFxfr2LFjeumll/T8888rNzdXkrRu3Trdc889euutt3TTTTcF9yABAADgKGENwIFqbW2Vy+XSyJEjJUm1tbXq7u7W9OnT/W3Gjh2r9PR0VVdXq7i4WNXV1UpMTPSHX0nKy8tTYmKiqqurAw7Al64sX2l7X/sRWtTDeaiJs1APZ6EezjKc6+F2uyVJlhXacYfyWAZSj4GOc80E4M7OTm3atElz5sxRQkKCJKmpqUlRUVFKSkrq1TYlJUVNTU3+NqNHj76sv9GjR/vbBKKmpmZI+xFa1MN5qImzUA9noR7OMtzqERcXp6ysLLW1tam1tTUkY7bFuyRJdXV1am9vH1JfdtbjmgjA3d3devDBB2VZlh5//PF+21sD+FhjWZZcLlfAc8nJyfF/erqUz+dTTU1Nn/sRWtTDeaiJs1APZ6EezjLc6xEfH6+EhNAsAcfHx0uSMjMzB91HIPW42LY/jg/A3d3dWrZsmRoaGvSzn/3Mv/ornV/p7e7uVnNzc69VYK/Xq/z8fH8br9d7Wb8ffvjhFVeG++N2u6/64Pe3H6FFPZyHmjgL9XAW6uEsw7keg1gDHBI7Hkc76+Ho6wBfDL/vvPOOfvrTnyo5ObnX/uzsbEVFRenQoUP+badOnVJ9fb0/AOfn56ulpUWvvfaav82rr76qlpYWfxvH6um5NvoEAAC4hoR1Bfjs2bN69913/bcbGhp05MgRJSUlacyYMXrggQf0xhtvqKKiQj6fT42NjZKkpKQkRUdHKzExUfPmzZPH41FycrKSkpLk8XiUkZHhvyrEzTffrOLiYj366KNau3atJKm0tFQzZ850/hUgIiKkP2ySPnrPnv5GjZf+4WF7+gIAALhGhTUA19bWauHChf7bZWVlkqS5c+fqO9/5jg4ePChJuvPOO3vdb8eOHSooKJAkrVq1SpGRkVq2bJk6Ojo0bdo0lZeX91oi37Rpk9avX69vfOMbkqRZs2Zd8drDjvTRe9KHx8I9CwAAgGEjrAG4oKBAdXV1fe6/2r6LYmJiVFpaqtLS0j7bjBo1Sps2bRrUHAEAADC8OPocYAAAAMBuBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYJawBuKqqSiUlJSoqKlJmZqZefPHFXvsty9KWLVtUVFSkyZMna8GCBaqvr+/VpqurS+vWrVNBQYHy8vJUUlKiDz74oFeb5uZmLV++XFOmTNGUKVO0fPlynTlzJujHBwAAAOcJawBua2tTZmamVq9efcX927ZtU2VlpVavXq09e/YoJSVFixYtUmtrq7/Nhg0bdODAAW3evFk7d+5UW1ublixZIp/P52/z0EMP6ejRo9q+fbu2b9+uo0ePasWKFUE/PgAAADhPZDgHnzFjhmbMmHHFfZZlaceOHSopKdHs2bMlSR6PR4WFhdq3b5/mz5+vlpYW7d27V0888YQKCwslSRs3btRtt92mw4cPq7i4WMeOHdNLL72k559/Xrm5uZKkdevW6Z577tFbb72lm266KTQHCwAAAEcIawC+moaGBjU2NqqoqMi/LTo6WlOnTlV1dbXmz5+v2tpadXd3a/r06f42Y8eOVXp6uqqrq1VcXKzq6molJib6w68k5eXlKTExUdXV1QEH4EtXlq+0va/9g+F2u2XJkizLph4tuWTvHJ0qGPXA0FATZ6EezkI9nGU418PtdkuyMVoM0FAey0DqMdBxHBuAGxsbJUmjR4/utT0lJUUnTpyQJDU1NSkqKkpJSUmXtWlqavK3+fs+LvZ7sU0gampqhrR/oOLi4pSVlaW2tjb1XHLKx1BExLVphKS6ujq1t7fb0qfT2VUP2IeaOAv1cBbq4SzDrR6XZotWm7JFf9riXZLsyR521sOxAfgil8vV67Y1gI8sA23z930PRE5Ojv/T06V8Pp9qamr63D9Y8fHxUkKCXZ1JkjIzM+3pz8GCVQ8MHjVxFurhLNTDWYZ7PeLj45WQEJol4Hgbskcg9bjYtj+ODcCpqamSzq/gjhkzxr/d6/UqJSVF0vmV3u7ubjU3N/daBfZ6vcrPz/e38Xq9l/X/4YcfXnFluD9ut/uqD35/+wPlkksaRFDvqzdJw/LJ3Be764GhoybOQj2chXo4y3Cuh23RYoDseBztrIdjrwM8btw4paam6tChQ/5tXV1dqqqq8ofb7OxsRUVF9Wpz6tQp1dfX+9vk5+erpaVFr732mr/Nq6++qpaWFn8bAAAAmCOsK8Bnz57Vu+++67/d0NCgI0eOKCkpSWlpaVq4cKEqKio0ceJETZgwQRUVFYqNjdWcOXMkSYmJiZo3b548Ho+Sk5OVlJQkj8ejjIwM/1Uhbr75ZhUXF+vRRx/V2rVrJUmlpaWaOXMmV4AAAAAwUFgDcG1trRYuXOi/XVZWJkmaO3euysvLtXjxYnV2dmrNmjVqbm5Wbm6unnnmGSVcck7sqlWrFBkZqWXLlqmjo0PTpk1TeXl5ryXyTZs2af369frGN74hSZo1a1af1x4GAADA8BbWAFxQUKC6uro+97tcLi1dulRLly7ts01MTIxKS0tVWlraZ5tRo0Zp06ZNQ5orAAAAhgfHngMMAAAABAMBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAACBMenqscE/BSJHhngAAAICpIiJc2nKwXsdPtwd9rLzxozT/MzcGfZxrAQEYAAAgjI6fbtfb3rNBHydtVFzQx7hWcAoEAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAA4O/ExcWFewoIoshwTwAAAMApenosud1uZWVlhXsqCCICMAAAwAURES5t+a961Z/wKj4+Pqhj5Y0fpfmfuTGoY+DKCMAAAACXOP5Ru46dalFCgiWXK3jjpI3iNItw4RxgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRBhWAb7/9dp0+ffqy7WfOnNHtt98+5EkBAAAAwTKoAHz8+HH19PRctr2rq0snT54c8qQAAACAYIkMpPF//dd/+f//pZdeUmJiov92T0+PXn75Zd1www22Te7cuXPasmWLfv3rX6upqUmpqamaO3eu7r//fkVEnM/ulmVp69ateu6553TmzBnl5uZq9erVSk9P9/fT1dUlj8ejffv2qbOzU7feeqsef/xxXX/99bbNFQAAANeGgALwt7/9bUmSy+XSypUre3cUGakbbrjhsu1DsW3bNu3evVsej0eTJk1SbW2tvv/97ysxMVH33Xefv01lZaXKy8s1ceJEPfXUU1q0aJF++9vfKiEhQZK0YcMG/fd//7c2b96sUaNGqby8XEuWLNELL7wgt9tt23wBAADgfAEF4KNHj0qSZs2apT179ui6664LyqQu+stf/qLbb79dt912myRp3Lhx2r9/v2prayWdX/3dsWOHSkpKNHv2bEmSx+NRYWGh9u3bp/nz56ulpUV79+7VE088ocLCQknSxo0bddttt+nw4cMqLi4O6jEAAADAWQIKwBcdPHjQ7nlc0ZQpU7R792797W9/08c//nEdPXpU//u//6tVq1ZJkhoaGtTY2KiioiL/faKjozV16lRVV1dr/vz5qq2tVXd3t6ZPn+5vM3bsWKWnp6u6ujrgAOzz+a66va/9g+F2u2XJkizLph4tuWTvHJ0qGPXA0FATZ6EezkI9nMPtdv/f+65lXXjnDD7b3uodOt5QfrYDeX4MdJxBBWBJevnll/Xyyy/L6/Ve9oW4srKywXbby+LFi9XS0qIvfOELcrvd8vl8evDBBzVnzhxJUmNjoyRp9OjRve6XkpKiEydOSJKampoUFRWlpKSky9o0NTUFPKeampoh7R+ouLg4ZWVlqa2tTT2trbb0GRHXphGS6urq1N7ebkufTmdXPWAfauIs1MNZqEd4+d97L7xHtp49G9TxOjpGXPi3Q602vdc7aTxJaos//wHCjuxh5/NjUAF469at+tGPfqTs7GylpqbK5QrOp6Pf/OY3+o//+A/94Ac/0KRJk3TkyBGVlZVpzJgxmjt3rr/d349vDeBjzUDaXElOTs4Vzxv2+Xyqqanpc/9gxcfHSxfOZbahM0lSZmamPf05WLDqgcGjJs5CPZyFejhLfFycpBYljBghBSnjSFJsbNyFf2OVkBD81f9QjyddyDEaWvYI5PlxsW1/BhWAd+/erbKyMn3lK18ZzN0H7IknntC//uu/6otf/KKk8w/eiRMnVFFRoblz5yo1NVXS+VXeMWPG+O/n9XqVkpIi6fxKb3d3t5qbm3utAnu9XuXn5wc8J7fbfdUHv7/9gXLJZeOT73w/Jr242l0PDB01cRbq4SzUwyEuvu+6XMHMv1ccMlRCPZ4dP9d2Pj8GdR3g7u5u3XLLLbZM4Go6OjouW911u93+1dtx48YpNTVVhw4d8u/v6upSVVWVP9xmZ2crKiqqV5tTp06pvr5+UAEYAAAA17ZBrQB/9atf1a9//Wv/ZdGCZebMmfrJT36itLQ0/ykQlZWVmjdvnqTzpz4sXLhQFRUVmjhxoiZMmKCKigrFxsb6zxNOTEzUvHnz5PF4lJycrKSkJHk8HmVkZPivCgEAAABzDCoAd3Z26vnnn9fLL7+szMxMRUb27ub73/++LZN79NFH9cMf/lBr1qyR1+vVmDFjdM899/QK3osXL1ZnZ6fWrFmj5uZm5ebm6plnnvFfA1iSVq1apcjISC1btkwdHR2aNm2aysvL+TUTAACAgQYVgOvq6vSJT3xCkvTmm2/22mfnF+ISEhL0yCOP6JFHHumzjcvl0tKlS7V06dI+28TExKi0tFSlpaW2zQ0AAADXpkEF4J///Od2zwMAAAAIiUF9CQ4AAAC4Vg1qBXjBggVXPdVhx44dg54QAAAAEEyDCsCf/OQne90+d+6cjhw5ovr6+qBfGxgAAAAYikEF4FWrVl1x+5YtW9TW1jakCQEAAADBZOs5wF/+8pe1d+9eO7sEAAAAbGVrAK6urlZ0dLSdXQIAAAC2GtQpEN/5znd63bYsS42NjaqtrdX9999vy8QAAACAYBhUAE5MTOx12+Vy6eMf/7geeOABFRUV2TIxAAAAIBgGFYDLysrsngcAAAAQEoMKwBfV1tbq2LFjcrlcmjRpkrKysuyaFwAAABAUgwrAXq9XDz74oF555RWNHDlSlmWppaVFBQUF2rx5s6677jq75wkAAADYYlBXgVi3bp1aW1u1f/9+vfLKK6qqqtK+ffvU2tqq9evX2z1HAAAAwDaDCsAvvfSSHn/8cd18883+bZMmTdJjjz2mP/zhD7ZNDgAAALDboAJwT0+PoqKiLtseGRmpnp6eIU8KAAAACJZBBeBbb71VGzZs0MmTJ/3bTp48qbKyMk2bNs22yQEAAAB2G9SX4FavXq37779ft99+u66//nq5XC69//77ysjI0MaNG+2eIwAAAGCbQQXgj33sY/rlL3+pQ4cO6a233pJlWZo0aZIKCwvtnh8AAABgq4BOgXj55Zd1xx13qLW1VZI0ffp0LViwQAsXLlROTo6++MUv6s9//nNQJgoAAADYIaAA/LOf/Ux33323EhISLtuXmJioe+65R5WVlbZNDgAAALBbQAG4rq5OxcXFfe6fPn26Xn/99SFPCgAAAAiWgAJwU1OTIiP7Pm04MjJSH3744ZAnBQAAAARLQAF47NixevPNN/vcX1dXp9TU1CFPCgAAAAiWgALwjBkz9G//9m/q7Oy8bF9HR4e2bNmimTNn2jY5AAAAwG4BXQbtW9/6ln73u9/pc5/7nO699159/OMfl8vl0rFjx7Rz5075fD6VlJQEa64AAADAkAUUgFNSUrR79249/vjjevLJJ2VZliTJ5XKpqKhIjz32mFJSUoIyUQAAAMAOAf8hjBtuuEHbtm1Tc3Oz3nnnHUnShAkTlJSUZPvkAAAAALsN6i/BSVJSUpImT55s51wAAACAoAvoS3AAAADAtY4ADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMfq6bHCPQUMQ4P+QxgAAADBFhHh0paD9Tp+uj3oY+WNH6X5n7kx6OMg/AjAAADA0Y6fbtfb3rNBHydtVFzQx4AzcAoEAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMIrjA/DJkyf18MMPq6CgQLm5ubrzzjtVW1vr329ZlrZs2aKioiJNnjxZCxYsUH19fa8+urq6tG7dOhUUFCgvL08lJSX64IMPQn0oAAAAcABHB+Dm5mZ9/etfV1RUlLZt26b9+/dr5cqVGjlypL/Ntm3bVFlZqdWrV2vPnj1KSUnRokWL1Nra6m+zYcMGHThwQJs3b9bOnTvV1tamJUuWyOfzheOwAAAAEEaODsDbtm3T9ddfr7KyMk2ePFnjxo3TtGnTdOONN0o6v/q7Y8cOlZSUaPbs2crIyJDH41FHR4f27dsnSWppadHevXu1cuVKFRYWKisrSxs3btSbb76pw4cPh/PwAAAAEAaR4Z7A1Rw8eFBFRUV64IEHVFVVpbFjx+qf/umfdPfdd0uSGhoa1NjYqKKiIv99oqOjNXXqVFVXV2v+/Pmqra1Vd3e3pk+f7m8zduxYpaenq7q6WsXFxQHNqa9V44vb7VxVdrvdsmRJlmVTj5ZcsneOThWMemBoqImzUA9noR59c7vdkmx8KxyQC4NZ1oV3zhCMGNLjC/14Q/nZDuT5MdBxHB2A33vvPe3atUuLFi1SSUmJXnvtNa1fv17R0dH6yle+osbGRknS6NGje90vJSVFJ06ckCQ1NTUpKipKSUlJl7VpamoKeE41NTVD2j9QcXFxysrKUltbm3ouOZ1jKCLi2jRCUl1dndrb223p0+nsqgfsQ02chXo4C/Xo7dL3wlab3guvpqNjxIV/OyVJrWfPhmi8jhAfX2jGk6S2+PMfIOzIHnY+PxwdgC3LUnZ2tv7f//t/kqSsrCz99a9/1a5du/SVr3zF387lcl12v4H0PRg5OTn+T6OX8vl8qqmp6XP/YMXHx0sJCXZ1JknKzMy0pz8HC1Y9MHjUxFmoh7NQj6uLj49XQkLwlyxjY+Mu/BsjSUoYMUJyBW8F+P/Gi1VCQvBX/0M9nnQhx2ho2SOQ58fFtv1xdABOTU3VzTff3GvbTTfdpP/8z//075fOr/KOGTPG38br9SolJUXS+ZXe7u5uNTc391oF9nq9ys/PD3hObrf7qg9+f/sD5ZLLxiff+X5MenG1ux4YOmriLNTDWahH34KYQ680mn/QUI0b2uML/Xh2/Fzb+fxw9JfgbrnlFv3tb3/rte3tt9/WDTfcIEkaN26cUlNTdejQIf/+rq4uVVVV+cNtdna2oqKierU5deqU6uvrBxWAAQAAcG1z9Arwfffdp69//ev6yU9+oi984Qt67bXX9Pzzz2vt2rWSzp/6sHDhQlVUVGjixImaMGGCKioqFBsbqzlz5kiSEhMTNW/ePHk8HiUnJyspKUkej0cZGRkqLCwM5+EBAAAgDBwdgCdPnqytW7fqySef1I9+9CONGzdOq1at0pe//GV/m8WLF6uzs1Nr1qxRc3OzcnNz9cwzzyjhkvNmV61apcjISC1btkwdHR2aNm2aysvL+TUTAACAgRwdgCVp5syZmjlzZp/7XS6Xli5dqqVLl/bZJiYmRqWlpSotLQ3GFAEAAHANcfQ5wAAAAIDdCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAIAB6+mxwj0FYMgiwz0BAABw7YiIcGnLwXodP90e9LHyxo/S/M/cGPRxYB4CMAAACMjx0+1623s26OOkjYoL+hgwE6dAAAAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAADAKARgAAAAGIUADAAAAKMQgAEAAGCUayoAV1RUKDMzUxs2bPBvsyxLW7ZsUVFRkSZPnqwFCxaovr6+1/26urq0bt06FRQUKC8vTyUlJfrggw9CPX0AAAA4wDUTgF977TU999xzyszM7LV927Ztqqys1OrVq7Vnzx6lpKRo0aJFam1t9bfZsGGDDhw4oM2bN2vnzp1qa2vTkiVL5PP5Qn0YAAAACLNrIgCfPXtWy5cv1/r165WUlOTfblmWduzYoZKSEs2ePVsZGRnyeDzq6OjQvn37JEktLS3au3evVq5cqcLCQmVlZWnjxo168803dfjw4XAdEgAAAMIkMtwTGIi1a9dqxowZKiws1FNPPeXf3tDQoMbGRhUVFfm3RUdHa+rUqaqurtb8+fNVW1ur7u5uTZ8+3d9m7NixSk9PV3V1tYqLiwOaS1+rxhe327mq7Ha7ZcmSLMumHi25ZO8cnSoY9cDQUBNnoR7OMpR6uFwRiohw2T2lftn21uTI8Sz/oOffOUMw4rB+PIf2WhPI82Og4zg+AO/fv19vvPGG9uzZc9m+xsZGSdLo0aN7bU9JSdGJEyckSU1NTYqKiuq1cnyxTVNTU8DzqampGdL+gYqLi1NWVpba2trUc8npHEMREdemEZLq6urU3t5uS59OZ1c9YB9q4izUw1kCrcfF94qN+2v03odngzSr/zNl4mjdV5yujo6OXqcaBktHx4gL/4Z6vE5JUuvZ4D6m4Tu+0IwnSW3x5z9A2JE97Hy9cnQAfv/997VhwwY988wziomJ6bOdy9X705k1gI81A2lzJTk5OXK73Zdt9/l8qqmp6XP/YMXHx0sJCXZ1JkmXnUc9HAWrHhg8auIs1MNZhloPb4elk23BX9Jr7jr/b2xsrBISgv/bg9jYuDCNdz5zJIwYIbmCtwIcvuMLzXjShRyjoWWPQJ4fF9v2x9EB+PXXX5fX69Vdd93l3+bz+VRVVaVnn31Wv/3tbyWdX+UdM2aMv43X61VKSoqk8yu93d3dam5u7rUK7PV6lZ+fH/Cc3G73VR/8/vYHyiWXjU++8/2Y9GZndz0wdNTEWaiHswylHkHMaQaO5/IPGqpxh/fjaU/2sPP1ytFfgrv11lv161//Wr/61a/8/2VnZ+tLX/qSfvWrX2n8+PFKTU3VoUOH/Pfp6upSVVWVP9xmZ2crKiqqV5tTp06pvr5+UAEYAAAA1zZHrwAnJCQoIyOj17b4+HiNGjXKv33hwoWqqKjQxIkTNWHCBFVUVCg2NlZz5syRJCUmJmrevHnyeDxKTk5WUlKSPB6PMjIyVFhYGPJjAgAAQHg5OgAPxOLFi9XZ2ak1a9aoublZubm5euaZZ5RwyXmzq1atUmRkpJYtW6aOjg5NmzZN5eXl/NoPAADAQNdcAP75z3/e67bL5dLSpUu1dOnSPu8TExOj0tJSlZaWBnt6AAAAcDhHnwMMAAAA2I0ADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAOAAcXFx4Z4CYIzIcE8AAIDhoqfHUkSEK+D7ud1uZWVlBWFGAK6EAAwAgE0iIlzacrBex0+3B3zftrY2xcfHB3SfvPGjNP8zNwY8FmA6AjAAADY6frpdb3vPBnQfy5JaW1uVkGDJFcACctooTpsABoNzgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwAAAADAKARgAAABGIQADAIatnh4r3FMA4ECR4Z4AAADBEhHh0paD9Tp+uj3oY+WNH6X5n7kx6OMAGDpHB+CKigr97ne/01tvvaXY2Fjl5+fr4Ycf1k033eRvY1mWtm7dqueee05nzpxRbm6uVq9erfT0dH+brq4ueTwe7du3T52dnbr11lv1+OOP6/rrrw/HYQEAQuj46Xa97T0b9HHSRsUFfQwA9nD0KRCvvPKK7r33Xj3//POqrKyUz+fTN7/5TbW1tfnbbNu2TZWVlVq9erX27NmjlJQULVq0SK2trf42GzZs0IEDB7R582bt3LlTbW1tWrJkiXw+XzgOCwAAAGHk6AD89NNP66677lJ6ero+8YlPqKysTCdOnNDrr78u6fzq744dO1RSUqLZs2crIyNDHo9HHR0d2rdvnySppaVFe/fu1cqVK1VYWKisrCxt3LhRb775pg4fPhzOwwMAAEAYOPoUiL/X0tIiSUpKSpIkNTQ0qLGxUUVFRf420dHRmjp1qqqrqzV//nzV1taqu7tb06dP97cZO3as0tPTVV1dreLi4oDm0Neq8cXtdq4qu91uWbIky64vcVhyyd45OlUw6oGhoSbOYko93G63JBtfRgco4PEu3sGyLrxSh2DMIRre4w29HgGPOKwfz6G91gTyejXQca6ZAGxZlsrKyjRlyhRlZGRIkhobGyVJo0eP7tU2JSVFJ06ckCQ1NTUpKirKH5ovbdPU1BTwPGpqaoa0f6Di4uKUlZWltrY29VxyOsdQRMS1aYSkuro6tbcH/wshTmBXPWAfauIsw7kel76Ottr0Ono1HR0jLvzbMejxWs8Gdq6yHWMy3pXG65QUeD0GP97wfDwlqS3+/AcIO7KHna9X10wAXrt2rd58803t3Lnzsn0uV+9PZ9YAPtYMpM2V5OTk+FcULuXz+VRTU9Pn/sGKj4+XEhLs6kySlJmZaU9/DhasemDwqImzmFSP+Ph4JSQEf7krNjbuwr+xSkgIcLXLstR69qwSRoyQXANfcRzSmINgzngxkhRwPQY/3vB8PKULOUZDyx6BvF5dbNufayIAr1u3TgcPHtQvfvGLXlduSE1NlXR+lXfMmDH+7V6vVykpKZLOr/R2d3erubm51yqw1+tVfn5+wHNxu91XffD72x8ol1w2PvnO9zPc3+wuZXc9MHTUxFlMqUcQM4wt4/l/ze5yDXquTj/Ga2u8odcj4BGH9eNpT/aw8/XK0V+CsyxLa9eu1e9+9zv97Gc/0/jx43vtHzdunFJTU3Xo0CH/tq6uLlVVVfnDbXZ2tqKionq1OXXqlOrr6wcVgAEAAHBtc/QK8Jo1a7Rv3z79+Mc/1ogRI/zn/CYmJio2NlYul0sLFy5URUWFJk6cqAkTJqiiokKxsbGaM2eOv+28efPk8XiUnJyspKQkeTweZWRkqLCwMJyHBwAAgDBwdADetWuXJGnBggW9tpeVlemuu+6SJC1evFidnZ1as2aNmpublZubq2eeeUYJl5w3u2rVKkVGRmrZsmXq6OjQtGnTVF5ebsSv/QAAANCbowNwXV1dv21cLpeWLl2qpUuX9tkmJiZGpaWlKi0ttXN6AAAAuAY5+hxgAEDwREVFyeUK7dtAT0+ILz4KAFfg6BVgAEDwREZGKiLCpS0H63X8dPCvDX5DcpyWzkoP+jgA0B8CMAAY7vjpdr3tDe4F/wHASTgFAgAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAhkRQXxWXQADgCV4EAAITEiJjQXnYtb/wozf/MjUEfB8C1hwAMAAipUF12LW1UXNDHAHBt4hQIAAAAGIUADAAAAKMQgAEAAGAUAjAAAACMQgAGAACAUQjAAAAAMAoBGAAAAEYhAAMAAMAoBGAAAAAYhQAMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAA4ChYmNjwz0FAAgLAjAAOEhPjxWScdxut2666aaQjAUAThMZ7gkAAP5PRIRLWw7W6/jp9qCP9ckxcbqvOD3o4wCA0xCAAcBhjp9u19ves0Edw7Kk6zgDAoChOAUCAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjEIABgAAgFEIwABwFaH6y2wAgNDhD2EAwFWE8i+z5Y0fpfmfuTHo4wCA6QjAANCPUPxlNklKGxUX9DEAAJwCAQAAAMMQgAEAAGAUAjAAAACMQgA2SVyy1NNjb5929wcAABBkfAnOJNEjpIgI6Q+bpI/eG3p/o8ZL//Dw0PsBAAAIIQKwiT56T/rwWLhnAQAAEBacAoHhLxinaXDqBwAA1yxWgDH82Xnah8SpHwAAXOMIwDADp30MCz09liIiXOGeBgDgGkcABnDNCOWfJZb408QAMFwRgDF4Fy+rFmHzqeQ29RkXF6Q/K+vw43YyO2oSqj9LLPGniQFguDIqAD/77LN6+umn1djYqPT0dK1atUqf/vSnwz2ta5fdl1WTpLFZ0tTFQ+7G7XYrKyvLhgldgYOPu5cQBOpATkmwqyb9hdKWjm55z3YNeRwAwPBlTAD+zW9+o7KyMj322GO65ZZbtHv3bi1evFj79+9XWlpauKfnCJZlyeUaxPmVgzy/9orjJY2zJVxastTW1qb49GK5ptw36H6uys7zim08bpdcwQnUktTjkyLc/ptDPR/X6vHJdUl/A/HA7elX3d/VfU5PHnhTH7WfG8rUJEmpiTGSpPHJ9q4EE9IBILyMCcCVlZWaN2+evva1r0mSHnnkEf3xj3/Url279NBDD/V7f8uyJEldXV1yuy9/w/b5fFfdPxhut1tnYtPUGWPPJbdiIlM00ue7Yp8x7giNjIvS6fZOnfMNvb/+RLql5LgYWbJ670i4Xi6fT5YVIbkG/+NpyZIVEaNzilCEzycracKQ+rviHIPR5xCP2yWXTrd3yn0uWiMtS2df2anu5lO2TDFybIYSPvU5f59RbpdGxETqdFuXzvVY/XcgyerpkevCqnTM2EyNzPm8zvzPs+psPtnvfWMiIzQyNuqq40WPnqik3C9p+ecyB35g/fD5fFpSPCHgoH413efOaXfVcZ3t7B3S08fEy+fz6ZbxIzV+VGxAfQb8AdayNCE5ps/xBv2B+CpuTo0b9PH15WrzHMzjOZTj7mu8AfVpWeroildsdKzUT9tL+xvKz8ylWjq71drR/4fG6xOj5PP5dOOoWEW6Bva8H4rwjRej7usTFRfbfz3sGW94Pp6SlJYUK5/P589JgxFIxrrY9mJu64vL6q/FMNDV1aW8vDz98Ic/1Gc/+1n/9vXr1+vo0aP6xS9+MaA+ampqgjlNAAAA2CAnJ0fR0dF97jdiBfj06dPy+XwaPXp0r+0pKSlqbGwcUB+RkZHKyclRRESE7asiAAAAGDrLstTT06PIyKtHXCMC8EV/H1wD+VVXRETEVT9JAAAA4NowvK+5dEFycrLcbreampp6bfd6vUpJSQnTrAAAABAORgTg6OhofepTn9KhQ4d6bT98+LDy8/PDNCsAAACEgzGnQCxatEgrVqxQdna28vPz9dxzz+n999/X/Pnzwz01AAAAhJAxAfiOO+7Q6dOn9eMf/1inTp1SRkaG/v3f/1033HBDuKcGAACAEDLiMmgAAADARUacAwwAAABcRAAGAACAUQjAAAAAMAoBGAAAAEYhANvgqaee0vz585Wbm6tPf/rTV2yTmZl52X+7du0K8UzNMJB6nDhxQiUlJcrLy1NBQYHWr1+vrq6uEM/UXLNmzbrs+bBp06ZwT8sYzz77rGbNmqWcnBzddddd+vOf/xzuKRlry5Ytlz0Xpk+fHu5pGaOqqkolJSUqKipSZmamXnzxxV77LcvSli1bVFRUpMmTJ2vBggWqr68P02yHv/7qsXLlysueL3ffffegxjLmMmjB1N3drc9//vPKy8vTnj17+mxXVlam4uJi/+3ExMRQTM84/dXD5/NpyZIlSk5O1s6dO/XRRx/pe9/7nizLUmlpaRhmbKYHHnig1wtXfHx8GGdjjt/85jcqKyvTY489pltuuUW7d+/W4sWLtX//fqWlpYV7ekZKT09XZWWl/7bb7Q7jbMzS1tamzMxM3XXXXVq6dOll+7dt26bKykqVl5dr4sSJeuqpp7Ro0SL99re/VUJCQhhmPLz1Vw9JKi4uVllZmf92VFTUoMYiANvggQcekCS98MILV203cuRIpaamhmJKRuuvHn/84x/117/+Vb///e81duxYSec/Va5cuVIPPvggL2ohMmLECJ4PYVBZWal58+bpa1/7miTpkUce0R//+Eft2rVLDz30UJhnZya3281zIUxmzJihGTNmXHGfZVnasWOHSkpKNHv2bEmSx+NRYWGh9u3bxx/SCoKr1eOi6OhoW54vnAIRQmvXrlVBQYHmzZunXbt2qaenJ9xTMtJf/vIXpaen+8OvJBUVFamrq0u1tbVhnJlZtm/froKCAt1555166qmnOAUlBLq6uvT666+rqKio1/bp06eruro6TLPCO++8o6KiIs2aNUsPPvig3nvvvXBPCZIaGhrU2NjY6/kSHR2tqVOn8nwJo1deeUXTpk3T5z73OT366KPyer2D6ocV4BD57ne/q2nTpik2NlYvv/yyPB6PTp8+rfvvvz/cUzNOU1OTUlJSem1LSkpSVFSUmpqawjQrsyxcuFBZWVkaOXKkampq9IMf/EANDQ3asGFDuKc2rJ0+fVo+n0+jR4/utT0lJUWNjY1hmpXZJk+eLI/Ho4kTJ8rr9fq/w7Bv3z4lJyeHe3pGu/icuNLz5cSJE+GYkvH+4R/+QZ///OeVlpamhoYG/fCHP9R9992nF154QdHR0QH1RQDuw5YtW7R169arttmzZ49ycnIG1N+lQfeTn/ykJOlHP/oRAXiA7K6Hy+UKaDv6F0iN/vmf/9m/7ROf+IRGjhypBx54QA8//DBv+iHw9z/nlmXxsx8mf//r3ry8PH32s5/Vr371Ky1atChMs8KlrvR8QXjccccd/v/PyMhQdna2Zs2apd///vf+01QGigDch3vvvbfXA30l48aNG3T/ubm5am1tveJqJC5nZz1SUlL06quv9trW3Nys7u7uyz7pY+CGUqO8vDxJ0rvvvksADqLk5GS53e7LftPh9Xp5HXKI+Ph4ZWRk6O233w73VIx38TzTpqYmjRkzxr+d54tzjBkzRmlpaYN6vhCA+3DdddfpuuuuC1r/R44cUUxMjEaOHBm0MYYTO+uRl5enn/zkJzp16pT/Re3QoUOKjo5Wdna2LWOYaCg1euONNySJLwIFWXR0tD71qU/p0KFD+uxnP+vffvjwYd1+++1hnBku6urq0rFjxzRlypRwT8V448aNU2pqqg4dOqSsrCxJ5+tTVVWlhx9+OMyzg3T+tK7333+/1weUgSIA2+DEiRNqbm7WiRMn5PP5dOTIEUnSjTfeqBEjRujgwYNqampSXl6eYmNj9ac//UmbN2/W3XffHfA5K+hff/UoKirSpEmTtGLFCq1YsULNzc3yeDy6++67uQJECFRXV+vVV19VQUGBEhISVFNTo7KyMs2aNYvLcIXAokWLtGLFCmVnZys/P1/PPfec3n//fb7RHiYej0czZ87Uxz72MX344Yd66qmn1Nraqrlz54Z7akY4e/as3n33Xf/thoYGHTlyRElJSUpLS9PChQtVUVGhiRMnasKECaqoqFBsbKzmzJkTxlkPX1erR1JSkrZu3arZs2crNTVVx48f1+bNm5WcnKx//Md/DHgsl8XJLEO2cuVK/fKXv7xs+44dO1RQUKA//OEPevLJJ/XOO+/IsiyNHz9eX/3qV3XvvfcqMpLPIHbrrx7S+ZC8Zs0a/c///I//xex73/seH0hC4PXXX9eaNWv01ltvqaurS2lpafriF7+of/mXf1FcXFy4p2eEZ599Vk8//bROnTqljIwMff/739fUqVPDPS0jPfjgg6qqqtJHH32k5ORk5eXl6bvf/a4mTZoU7qkZ4U9/+pMWLlx42fa5c+eqvLxclmVp69ateu6559Tc3Kzc3FytXr1aGRkZYZjt8He1ejz++OP69re/rTfeeEMtLS1KTU1VQUGBvvvd7+pjH/tYwGMRgAEAAGAUrgMMAAAAoxCAAQAAYBQCMAAAAIxCAAYAAIBRCMAAAAAwCgEYAAAARiEAAwAAwCgEYAAAABiFAAwAAACjEIABAABgFAIwAAAAjPL/AXnCf6OvPy1+AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_neg_scores_after_rank = []\n",
    "train_pos_scores_after_rank = []\n",
    "\n",
    "with open('./trainset/kg_merged.jsonl', 'r') as p_t_file:\n",
    "    lines = p_t_file.readlines()\n",
    "    pos_pair_list = []\n",
    "    for line in lines:\n",
    "        try:\n",
    "            json_obj = json.loads(line)\n",
    "        except Exception as e:\n",
    "            continue\n",
    "        query = json_obj['query'].strip()\n",
    "        pos = json_obj['pos'][0].strip()\n",
    "        pos_pair_list.append([query, pos])\n",
    "    pos_scores = reranker.compute_score(pos_pair_list)\n",
    "    train_pos_scores_after_rank.extend(pos_scores)\n",
    "\n",
    "with open('./trainset/refuse_trainset.jsonl', 'r') as n_t_file:\n",
    "    lines = n_t_file.readlines()\n",
    "    for line in lines:\n",
    "        obj = json.loads(line)\n",
    "        neg_pair_list = [ [obj['query'].strip(), neg.strip()] for neg in obj['neg'] ]\n",
    "        neg_scores = reranker.compute_score(neg_pair_list)\n",
    "        train_neg_scores_after_rank.extend(neg_scores)\n",
    "        \n",
    "plot_stat(train_pos_scores_after_rank, train_neg_scores_after_rank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "924abc93-252b-45c9-89a6-545c99de3fc3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compute Scores: 100%|██████████| 2/2 [00:00<00:00,  3.00it/s]\n"
     ]
    }
   ],
   "source": [
    "test_neg_scores_after_rank = []\n",
    "test_pos_scores_after_rank = []\n",
    "\n",
    "with open('./testset/kg_merged.jsonl', 'r') as p_t_file:\n",
    "    lines = p_t_file.readlines()\n",
    "    pos_pair_list = []\n",
    "    for line in lines:\n",
    "        try:\n",
    "            json_obj = json.loads(line)\n",
    "        except Exception as e:\n",
    "            continue\n",
    "        query = json_obj['query'].strip()\n",
    "        pos = json_obj['pos'][0].strip()\n",
    "        pos_pair_list.append([query, pos])\n",
    "    pos_scores = reranker.compute_score(pos_pair_list)\n",
    "    test_pos_scores_after_rank.extend(pos_scores)\n",
    "\n",
    "with open('./testset/refuse_testset.jsonl', 'r') as n_t_file:\n",
    "    lines = n_t_file.readlines()\n",
    "    for line in lines:\n",
    "        obj = json.loads(line)\n",
    "        neg_pair_list = [ [obj['query'].strip(), neg.strip()] for neg in obj['neg'] ]\n",
    "        neg_scores = reranker.compute_score(neg_pair_list)\n",
    "        test_neg_scores_after_rank.extend(neg_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4b0a4a79-922e-421f-94c5-c3e5d60b5f89",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArcAAAH5CAYAAACbJxb2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAygklEQVR4nO3df3RU9Z3/8ddkJpMfJMSYBDRAge+SxKYJJCibBoIoKvUo+0NoKd/jAZfTcsjqRqW1SmmDArIh2jXHDcqmqDmyCuoB6/e76HrUdd3WwBb2bNoTLA1Z/aINiCQR04QhJMzc7x9sZglJSGYyk3v55Pk4hyNz7yf38555Z8YXdz5zx2VZliUAAADAADF2FwAAAABECuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjOGxuwAnCAQCOn/+vGJiYuRyuewuBwAAAJewLEuBQEAej0cxMYOfnyXcSjp//rwaGhrsLgMAAABDyM/Pl9frHXQ/4VYKpv/8/Hy53e5++/1+vxoaGgbdj9FHT5yFfjgL/XAW+uEs9MN5htuT3nGXO2srEW4lKbgUwe12X/ZBHWo/Rh89cRb64Sz0w1noh7PQD+cZbk+GWkLKB8oAAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLd2CQTMmgcAAMABPHYXMGbFxEi//Jn01R+iN8dVU6QbH4re8QEAAByGcGunr/4gffmx3VUAAAAYg2UJAAAAMAbhFgAAAMawNdzu2rVLf/Znf6bZs2dr9uzZ+u53v6t/+7d/C+63LEvV1dUqKSnRzJkztWLFCjU1NfU5Rnd3tzZv3qyioiIVFBSotLRUJ0+eHO27AgAAAAewNdxec801euihh7R3717t3btX3/zmN3XfffcFA+yOHTtUW1urDRs2aM+ePUpPT9eqVavU2dkZPMaWLVv07rvvqqqqSrt27ZLP59OaNWvk9/vtulsAAACwia3hduHChVqwYIGmT5+u6dOna+3atUpMTNRvfvMbWZalnTt3qrS0VIsWLVJ2drYqKyvV1dWlffv2SZI6Ojq0d+9erVu3TnPnzlVubq6efPJJHT16VPv377fzrgEAAMAGjrlagt/v19tvvy2fz6fCwkI1NzerpaVFJSUlwTFer1dz5sxRfX29li9frsOHD6unp0fz5s0Ljpk4caKysrJUX1+v+fPnh1zD5bZH8myw2+2WJUuyrIgdsz9LLkW2bqeIRk8QPvrhLPTDWeiHs9AP5xluT4bbM9vDbWNjo5YvX65z584pMTFRzzzzjGbMmKH//M//lCSlpaX1GZ+enq4TJ05IklpbWxUbG6uUlJR+Y1pbW0OupaGhYUT7hyshIUG5ubny+XwKXLTEItJiEnwapwuP8dmzZ6M2j50i1RNEBv1wFvrhLPTDWeiH80SqJ7aH2+nTp+uNN97QH//4R73zzjt65JFH9NJLLwX3u1yuPuOtYZzpHM6YgeTn58vtdvfb7vf71dDQMOj+cCUmJkpJSRE73gATSJJycnKiN4dNotUThId+OAv9cBb64Sz0w3mG25PecUOxPdx6vV5NnTpV0oVw2dDQoJ07d2r16tWSLpydnTBhQnB8W1ub0tPTJV04Q9vT06P29vY+Z2/b2tpUWFgYci1ut/uyD+pQ+0Plkku6JLxH1oVjm/zkjXRPMDL0w1noh7PQD2ehH84TqZ447jq3lmWpu7tbkydPVkZGhurq6oL7uru7dejQoWBwzcvLU2xsbJ8xp06dUlNTU1jhFgAAAFc2W8/cPvXUU7rxxht1zTXX6MyZM3rrrbd08OBBPffcc3K5XFq5cqVqamo0bdo0TZ06VTU1NYqPj9fixYslScnJyVq6dKkqKyuVmpqqlJQUVVZWKjs7W3PnzrXzrgEAAMAGtobb1tZWPfzwwzp16pSSk5OVk5Oj5557Lnj1g9WrV+vcuXPauHGj2tvbNWvWLL3wwgtKumid6vr16+XxePTggw+qq6tLxcXF2rp1K281AAAAjEG2htu//du/vex+l8ulsrIylZWVDTomLi5O5eXlKi8vj3R5AAAAuMI4bs0tAAAAEC7CLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGsDXc1tTUaOnSpSosLFRxcbHuvfdeffLJJ33GrFu3Tjk5OX3+LFu2rM+Y7u5ubd68WUVFRSooKFBpaalOnjw5mncFAAAADmBruD148KDuvvtuvfbaa6qtrZXf79f3vvc9+Xy+PuPmz5+vDz/8MPjn5z//eZ/9W7Zs0bvvvquqqirt2rVLPp9Pa9askd/vH827AwAAAJt57Jz8+eef73O7oqJCxcXF+uijjzRnzpzgdq/Xq4yMjAGP0dHRob179+qJJ57Q3LlzJUlPPvmkbrrpJu3fv1/z58+P3h0AAACAo9gabi/V0dEhSUpJSemz/eDBgyouLtb48eM1Z84crV27VmlpaZKkw4cPq6enR/PmzQuOnzhxorKyslRfXx9SuB3sTG/v9kieCXa73bJkSZYVsWP2Z8mlyNbtFNHoCcJHP5yFfjgL/XAW+uE8w+3JcHvmmHBrWZYqKip0/fXXKzs7O7j9xhtv1O23367MzEw1Nzfr6aef1j333KPXX39dXq9Xra2tio2N7ReI09PT1draGlINDQ0NI9o/XAkJCcrNzZXP51OgszMixxxITIJP4yQ1Njbq7NmzUZvHTpHqCSKDfjgL/XAW+uEs9MN5ItUTx4TbTZs26ejRo9q1a1ef7XfccUfw79nZ2crLy9PChQv1wQcfaNGiRYMezwrjjGh+fr7cbne/7X6/Xw0NDYPuD1diYqKUlBSx4w0wgSQpJycnenPYJFo9QXjoh7PQD2ehH85CP5xnuD3pHTcUR4TbzZs36/3339dLL72ka6655rJjJ0yYoMzMTB07dkzShTO0PT09am9v73P2tq2tTYWFhSHV4Xa7L/ugDrU/VC65JJcrYscbaAZJRj95I90TjAz9cBb64Sz0w1noh/NEqie2Xi3Bsixt2rRJ77zzjl588UVNmTJlyJ85ffq0Pv/8c02YMEGSlJeXp9jYWNXV1QXHnDp1Sk1NTSGHWwAAAFzZbD1zu3HjRu3bt0/PPvusxo0bp5aWFklScnKy4uPjdebMGW3btk2LFi1SRkaGjh8/rqqqKqWmpurWW28Njl26dKkqKyuVmpqqlJQUVVZWKjs7O3j1BAAAAIwNtobb3bt3S5JWrFjRZ3tFRYWWLFkit9uto0eP6o033lBHR4cyMjJUVFSkqqoqJV20VnX9+vXyeDx68MEH1dXVpeLiYm3dupW3GwAAAMYYW8NtY2PjZffHx8f3uxbuQOLi4lReXq7y8vJIlQYAAIArkK1rbgEAAIBIItwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAxbw21NTY2WLl2qwsJCFRcX695779Unn3zSZ4xlWaqurlZJSYlmzpypFStWqKmpqc+Y7u5ubd68WUVFRSooKFBpaalOnjw5mncFAAAADmBruD148KDuvvtuvfbaa6qtrZXf79f3vvc9+Xy+4JgdO3aotrZWGzZs0J49e5Senq5Vq1aps7MzOGbLli169913VVVVpV27dsnn82nNmjXy+/123C0AAADYxNZw+/zzz2vJkiXKysrSddddp4qKCp04cUIfffSRpAtnbXfu3KnS0lItWrRI2dnZqqysVFdXl/bt2ydJ6ujo0N69e7Vu3TrNnTtXubm5evLJJ3X06FHt37/fzrsHAACAUeaxu4CLdXR0SJJSUlIkSc3NzWppaVFJSUlwjNfr1Zw5c1RfX6/ly5fr8OHD6unp0bx584JjJk6cqKysLNXX12v+/PnDnn+wM7292yN5JtjtdsuSJVlWxI7ZnyWXIlu3U0SjJwgf/XAW+uEs9MNZ6IfzDLcnw+2ZY8KtZVmqqKjQ9ddfr+zsbElSS0uLJCktLa3P2PT0dJ04cUKS1NraqtjY2GAgvnhMa2trSDU0NDSMaP9wJSQkKDc3Vz6fT4GLlldEWkyCT+MkNTY26uzZs1Gbx06R6gkig344C/1wFvrhLPTDeSLVE8eE202bNuno0aPatWtXv30ul6vPbWsYZzuHM+ZS+fn5crvd/bb7/X41NDQMuj9ciYmJUlJSxI43wASSpJycnOjNYZNo9QThoR/OQj+chX44C/1wnuH2pHfcUBwRbjdv3qz3339fL730kq655prg9oyMDEkXzs5OmDAhuL2trU3p6emSLpyh7enpUXt7e5+zt21tbSosLAypDrfbfdkHdaj9oXLJJV0S3CPrwrFNfvJGuicYGfrhLPTDWeiHs9AP54lUT2z9QJllWdq0aZPeeecdvfjii5oyZUqf/ZMnT1ZGRobq6uqC27q7u3Xo0KFgcM3Ly1NsbGyfMadOnVJTU1PI4RYAAABXNlvP3G7cuFH79u3Ts88+q3HjxgXX2CYnJys+Pl4ul0srV65UTU2Npk2bpqlTp6qmpkbx8fFavHhxcOzSpUtVWVmp1NRUpaSkqLKyUtnZ2Zo7d66ddw8AAACjzNZwu3v3bknSihUr+myvqKjQkiVLJEmrV6/WuXPntHHjRrW3t2vWrFl64YUXlHTRWtX169fL4/HowQcfVFdXl4qLi7V161bebgAAABhjbA23jY2NQ45xuVwqKytTWVnZoGPi4uJUXl6u8vLySJYHAACAK4yta24BAACASCLcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADBGWOH2lltu0enTp/tt/+Mf/6hbbrllxEUBAAAA4Qgr3B4/flyBQKDf9u7ubn3xxRcjLgoAAAAIhyeUwf/yL/8S/PuvfvUrJScnB28HAgEdOHBAkyZNilx1AAAAQAhCCrf33XefJMnlcmndunV9D+TxaNKkSf22AwAAAKMlpHD7+9//XpK0cOFC7dmzR1dffXVUigIAAADCEVK47fX+++9Hug4AAABgxMIKt5J04MABHThwQG1tbf0+XFZRUTHiwgAAAIBQhRVut23bpmeeeUZ5eXnKyMiQy+WKdF0AAABAyMIKt6+88ooqKir0l3/5lxEuBwAAAAhfWNe57enp0ezZsyNdCwAAADAiYYXbb3/72/qnf/qnSNcCAAAAjEhYyxLOnTun1157TQcOHFBOTo48nr6H+fGPfxyR4gAAAIBQhBVuGxsbdd1110mSjh492mcfHy4DAACAXcIKt//4j/8Y6ToAAACAEQtrzS0AAADgRGGduV2xYsVllx/s3Lkz7IIAAACAcIUVbr/+9a/3uX3+/HkdOXJETU1NXPsWAAAAtgkr3K5fv37A7dXV1fL5fCMqCAAAAAhXRNfc/vmf/7n27t0byUMCAAAAwxbRcFtfXy+v1xvJQwIAAADDFtayhL/5m7/pc9uyLLW0tOjw4cO69957I1IYAAAAEKqwwm1ycnKf2y6XS9OnT9f999+vkpKSiBQGAAAAhCqscFtRURHpOgAAAIARCyvc9jp8+LA+/vhjuVwuzZgxQ7m5uZGqCwAAAAhZWOG2ra1Na9eu1cGDBzV+/HhZlqWOjg4VFRWpqqpKV199daTrBAAAAIYU1tUSNm/erM7OTr355ps6ePCgDh06pH379qmzs1OPP/54pGsEAAAAhiWscPurX/1Kjz32mP7kT/4kuG3GjBl69NFH9ctf/jJixQEAAAChCCvcBgIBxcbG9tvu8XgUCARGXBQAAAAQjrDC7Te/+U1t2bJFX3zxRXDbF198oYqKChUXF0esOAAAACAUYYXbDRs26MyZM7rlllt066236rbbbtMtt9yiM2fOqLy8fNjHOXTokEpLS1VSUqKcnBy99957ffavW7dOOTk5ff4sW7asz5ju7m5t3rxZRUVFKigoUGlpqU6ePBnO3QIAAMAVLqyrJVx77bX6xS9+obq6On3yySeyLEszZszQ3LlzQzqOz+dTTk6OlixZorKysgHHzJ8/v891dS9dDrFlyxb967/+q6qqqnTVVVdp69atWrNmjV5//XW53e7Q7xwAAACuWCGF2wMHDmjz5s167bXXlJSUpHnz5mnevHmSpI6ODt15553auHGjbrjhhmEdb8GCBVqwYMFlx3i9XmVkZAy4r6OjQ3v37tUTTzwRDNZPPvmkbrrpJu3fv1/z588P4d4BAADgShdSuH3xxRe1bNkyJSUl9duXnJys7373u6qtrR12uB2OgwcPqri4WOPHj9ecOXO0du1apaWlSbrwJRI9PT3BgC1JEydOVFZWlurr60MOt36//7LbB9sfDrfbLUuWZFkRO2Z/llyKbN1OEY2eIHz0w1noh7PQD2ehH84z3J4Mt2chhdvGxkb96Ec/GnT/vHnz9MILL4RyyMu68cYbdfvttyszM1PNzc16+umndc899+j111+X1+tVa2urYmNjlZKS0ufn0tPT1draGvJ8DQ0NI9o/XAkJCcrNzZXP51OgszMixxxITIJP43Shb2fPno3aPHaKVE8QGfTDWeiHs9APZ6EfzhOpnoQUbltbW+XxDP4jHo9HX3755YiL6nXHHXcE/56dna28vDwtXLhQH3zwgRYtWjToz1lhng3Nz88fcJ2u3+9XQ0PDoPvDlZiYKA1wFjyCE0iScnJyojeHTaLVE4SHfjgL/XAW+uEs9MN5htuT3nFDCSncTpw4UUePHtXUqVMH3N/Y2Djo+thImDBhgjIzM3Xs2DFJF87Q9vT0qL29vc/Z27a2NhUWFoZ8fLfbfdkHdaj9oXLJJblcETveQDNIMvrJG+meYGToh7PQD2ehH85CP5wnUj0J6VJgCxYs0N///d/r3Llz/fZ1dXWpurpaN99884iLGszp06f1+eefa8KECZKkvLw8xcbGqq6uLjjm1KlTampqCivcAgAA4MoW0pnbv/7rv9Y777yjb33rW7r77rs1ffp0uVwuffzxx9q1a5f8fr9KS0uHfbwzZ87os88+C95ubm7WkSNHlJKSopSUFG3btk2LFi1SRkaGjh8/rqqqKqWmpurWW2+VdOFDbEuXLlVlZaVSU1OVkpKiyspKZWdnh3xZMgAAAFz5Qgq36enpeuWVV/TYY4/pqaeeCq5tdblcKikp0aOPPqr09PRhH+/w4cNauXJl8Hbv9WzvuusuPfbYYzp69KjeeOMNdXR0KCMjQ0VFRaqqqupztYb169fL4/HowQcfVFdXl4qLi7V161beagAAABiDQv4Sh0mTJmnHjh1qb2/Xp59+KkmaOnVqvysWDEdRUZEaGxsH3f/8888PeYy4uDiVl5eH9M1oAAAAMFNY31AmSSkpKZo5c2YkawEAAABGJKQPlAEAAABORrgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxrA13B46dEilpaUqKSlRTk6O3nvvvT77LctSdXW1SkpKNHPmTK1YsUJNTU19xnR3d2vz5s0qKipSQUGBSktLdfLkydG8GwAAAHAIW8Otz+dTTk6ONmzYMOD+HTt2qLa2Vhs2bNCePXuUnp6uVatWqbOzMzhmy5Ytevfdd1VVVaVdu3bJ5/NpzZo18vv9o3U3AAAA4BAeOydfsGCBFixYMOA+y7K0c+dOlZaWatGiRZKkyspKzZ07V/v27dPy5cvV0dGhvXv36oknntDcuXMlSU8++aRuuukm7d+/X/Pnzw+pnsECce/2SAZmt9stS5ZkWRE7Zn+WXIps3U4RjZ4gfPTDWeiHs9APZ6EfzjPcngy3Z7aG28tpbm5WS0uLSkpKgtu8Xq/mzJmj+vp6LV++XIcPH1ZPT4/mzZsXHDNx4kRlZWWpvr4+5HDb0NAwov3DlZCQoNzcXPl8PgUuOgsdaTEJPo2T1NjYqLNnz0ZtHjtFqieIDPrhLPTDWeiHs9AP54lUTxwbbltaWiRJaWlpfbanp6frxIkTkqTW1lbFxsYqJSWl35jW1taQ58zPz5fb7e633e/3q6GhYdD94UpMTJSSkiJ2vAEmkCTl5OREbw6bRKsnCA/9cBb64Sz0w1noh/MMtye944bi2HDby+Vy9bltDeNt/OGMGYjb7b7sgzrU/lC55JIuuX+RdeHYJj95I90TjAz9cBb64Sz0w1noh/NEqieOvRRYRkaGJPU7A9vW1qb09HRJF87Q9vT0qL29fdAxAAAAGDscG24nT56sjIwM1dXVBbd1d3fr0KFDKiwslCTl5eUpNja2z5hTp06pqakpOAYAAABjh63LEs6cOaPPPvsseLu5uVlHjhxRSkqKMjMztXLlStXU1GjatGmaOnWqampqFB8fr8WLF0uSkpOTtXTpUlVWVio1NVUpKSmqrKxUdnZ28OoJAAAAGDtsDbeHDx/WypUrg7crKiokSXfddZe2bt2q1atX69y5c9q4caPa29s1a9YsvfDCC0q66ENY69evl8fj0YMPPqiuri4VFxdr69atrKMBAAAYg2wNt0VFRWpsbBx0v8vlUllZmcrKygYdExcXp/LycpWXl0ejRAAAAFxBHLvmFgAAAAgV4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAADAEIGAZfR8w+GxuwAAAABERkyMS9XvN+n46bNRn2tSaoLKFmZFfZ5QEW4BAAAMcvz0WR1rO2N3GbZhWQIAAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuAUAAIAxCLcAAAAwBuEWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjOHocFtdXa2cnJw+f+bNmxfcb1mWqqurVVJSopkzZ2rFihVqamqysWIAAADYyWN3AUPJyspSbW1t8Lbb7Q7+fceOHaqtrdXWrVs1bdo0bd++XatWrdLbb7+tpKQkO8oFAACAjRwfbt1utzIyMvpttyxLO3fuVGlpqRYtWiRJqqys1Ny5c7Vv3z4tX7485Ln8fv9ltw+2Pxxut1uWLMmyInbM/iy5FNm6nSIaPUH46Iez0A9noR/OYno/ek8CRjVeXGKkj+VwezLceRwfbj/99FOVlJTI6/Vq1qxZ+sEPfqApU6aoublZLS0tKikpCY71er2aM2eO6uvrwwq3DQ0NI9o/XAkJCcrNzZXP51OgszMixxxITIJP4yQ1Njbq7NmzUZvHTpHqCSKDfjgL/XAW+uEsJvbj4nzRGcV80cuX6JIUuZwRqZ44OtzOnDlTlZWVmjZtmtra2rR9+3YtX75c+/btU0tLiyQpLS2tz8+kp6frxIkTYc2Xn5/fZ9lDL7/fr4aGhkH3hysxMVGK5vKJxERJUk5OTvTmsEm0eoLw0A9noR/OQj+cZSz0IzExUUlJ0T91mxihnDHcnvSOG4qjw+2CBQv63C4oKNBtt92mN954Q7NmzZIkuVyuPmOsEZyHd7vdl31Qh9ofKpdc0iX1R9aFY5v65JUi3xOMDP1wFvrhLPTDWUzvR1TjxSUi9ThGqieOvlrCpRITE5Wdna1jx44F1+G2trb2GdPW1qb09HQ7ygMAAIDNrqhw293drY8//lgZGRmaPHmyMjIyVFdX12f/oUOHVFhYaGOVAAAAsIujlyVUVlbq5ptv1rXXXqsvv/xS27dvV2dnp+666y65XC6tXLlSNTU1mjZtmqZOnaqamhrFx8dr8eLFdpcOAAAAGzg63J48eVI/+MEP9NVXXyk1NVUFBQV67bXXNGnSJEnS6tWrde7cOW3cuFHt7e2aNWuWXnjhBa5xCwAAMEY5OtxWVVVddr/L5VJZWZnKyspGqSIAAAA42RW15hYAAAC4HMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAACMQbgFAACAMQi3AAAAMAbhFgAAAMYg3AIAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAAIxBuB0jAgHL7hIAABhT+H+vPTx2FzCWfdHRpc6Wzqgd3+vyaYqkVw5+puV/+rWozQMAAPqLiXGp+v0mHT99dlTmK5hyFf+/F+HWVt3nA+rq8UdvgvMBSdKpjnPRmwMAAAzq+OmzOtZ2ZlTmyrwqYVTmcTqWJQAAAMAYhFsAAAAYg3ALAAAAYxBuAQAAYAzCLQAAsMVoXyqLS3ONDVwtAQAA2GI0L5U1KTVBZQuz5I/iRYrgDIRbAABgm9G8VBbGBpYlAAAAwBiEWwAAMKYkJPBlByZjWQIAADBeSkKsAgFLbrdbubm5dpeDKCLcAgAA442L81z4ANu/NKnpRJsSExOjOl/BlKu0/E+/FtU5MDDCLQAAGDOOf3VWH5/qUFKSJZcrevNkXsXSB7uw5hYAAADGINwCAADAGIRbAAAAGINwCwAAAGMQbgEAAGAMwi0AAA4VCFhGzwdEA5cCAwDAoWJiXKp+v0nHT5+N+lyTUhNUtjAr6vMA0Ua4BQDAwY6fPqtjbWfsLgO4YrAsAQAABL+eFrjSceYWAIAoS0hw/rdVBb+edpSWQfD1tIgWwi0AAMMQCFiKiQn9+1rdbrdyc3OjUFF0jNYyCL6eFtFCuAUAYBhGclbT5/MpMTExpJ/hzCYQHsItAADDFM5ZTcuSOjs7lZRkyRXCiV/ObALh4QNlY0BGcpzdJQAAAIwKwq3BPOOulgIB/e9ov60VCET3+ECUcIF8ADAPyxIMFhOfJMXEqO2ftyit51R0JrlqinTjQ9E5NhBlXCAfAMxDuB0Dzn/5mdRz3O4yAEcy9QL5V8KlpwAgGliWAACjYDQvkN976SmXy+yXeJZ5ABgIZ24BYBSM9gXy0+Jd+tGd+VGf52LhXgc2XHzhAICBEG4BYBSNxjIIy5J8iaMXMnvZETb5wgEAlyLcApcKBKSYKL+dG/BLMe7oziGNzn2BI6Umekf9TKrEt1sBsB/hFrhUTIz0y59JX/0hOseffL00e2V055C4ksUYNy5+dJdB8LY9AKcg3OKKFdVPg3/1B+nLj6Nz7JTJ0Z8D+G+cSQUw1hBuceW46C323k+DAwAAXIxwiyvHRcsFLFny+XxKTEyUSxFcU9i7ZAAAAFyRCLe4svS+lW9ZCnR2SklJkiuC4bZ3yQBCxpcGAACcgI9RA2GyrNG9gHzI8yWkXljKMQrcLpdyc3Pldod/BQguyA8AiATO3I4BMS6XLMuSK5JnOIcwGpcgirH5Elcul0vNp306dz60ADkurUvXSPqio0udLZ3D+pmkOI8mjo8Pab5xaW5dExOjtn/eIt+pYyHVmOh1K21cnL7o6FL3EPN506Zq4uKf6sVfNam9O6RpgqalJWrxzMzwfvgKlDbOq+T42KjOMXE8Z9KBUEXyuZmRHCdp4A9bdnT1qO1MmC+YGJIx4fbll1/W888/r5aWFmVlZWn9+vW64YYb7C7LEWJc4QexoSQkxStzgGupRjvYulwujRs3rs+20Q7wknTufEBdPf6Qfsbrv9CD7hB+Ns4TE/J8vfP4Th3TH4//PqQaXQmxUmqiOls6h5wvKe4qKRDQPfOzQpojVD3nz+tn7zSp/WxPRI5n16WrUhJitenPc+WNHYWX30BAKQnRDdGAKdLGeVW1bFbEn5v339L/tbG757zWvvZbAm6UGBFu33rrLVVUVOjRRx/V7Nmz9corr2j16tV68803lZk5ds4GDSWcIDYUrydRionRF/seV+cX/0+SlBTv0cTk+GGd9RuuxOlFSrvx+33Odgb8fsX899vgcZ4YTU5NjMhcCE1MfJIUE6OT/3eTzrR8GtYxen9n/nDaN+DvTFz6VE3+iw1qP9sTscta2XXpqnFxHnljPWr+P5t0rjW8x2s4Uq6ZrvQ7f6pxcUa8zANRlxwfG9Hn5mCva72vZ8nxsYTbKDHiVa+2tlZLly7Vd77zHUnST37yE3344YfavXu3fvjDHw75871rGbu7uwdcM+j3+y+7Pxxut1vxE/9EgZjonVWJv3qK/H6/POn/S/6uJCXGdckT4TO3vXPEeDzyeC+8BRPjccvvilUgxq9ATGTmCyjmwjxp0+UJXOiBFQjI9d9njGNj3To/zjvyKyckXSv5/dJV0yTX0L0J5zHtfcxC6X+c1y1/YlxI84UzTzjz/c/vgDf4OxCq3t8ZjzduwN8Zt8crv9+vwinjNeWq+LDmuFTWhET5/X7NvuSY0XoHoHe+6/77v+4RPF7D4XJ7gvO5lRa1eaQLj1n2II9nJOe4uC+D9S/S80RkPstSV3ei4r3xwQ/ADuf3bKT3MdTf5XDnC/c5E+p8I31u/s98ycpIsPr0Q5LSk70RfW4O9rrW+3o25ap4eVyR/azBNcmx8vv9+loUjj2QzJR4+f3+YE4K13BzVu+4oT6D4rJG+1MxEdbd3a2CggI9/fTTuu2224LbH3/8cf3+97/XSy+9NKxjNDQ0RLNMAAAAREB+fr68Xu+g+6/4M7enT5+W3+9XWlrfMxPp6elqaWkZ1jE8Ho/y8/MVExMz6ms2AQAAMDTLshQIBOTxXD6+XvHhtteloTSUty9iYmIu+y8AAAAAXBmu+Ovcpqamyu12q7W1tc/2trY2paen21QVAAAA7HDFh1uv16tvfOMbqqur67N9//79KiwstKkqAAAA2MGIZQmrVq3Sww8/rLy8PBUWFurVV1/V559/ruXLl9tdGgAAAEaREeH2jjvu0OnTp/Xss8/q1KlTys7O1s9//nNNmjTJ7tIAAAAwiq74S4EBAAAAva74NbcAAABAL8ItAAAAjEG4BQAAgDEItwAAADAG4XYI27dv1/LlyzVr1izdcMMNA47Jycnp92f37t2jXOnYMJx+nDhxQqWlpSooKFBRUZEef/xxdXd3j3KlY9fChQv7PR9+9rOf2V3WmPHyyy9r4cKFys/P15IlS/Qf//Efdpc0ZlVXV/d7LsybN8/ussaMQ4cOqbS0VCUlJcrJydF7773XZ79lWaqurlZJSYlmzpypFStWqKmpyaZqzTdUP9atW9fv+bJs2bKw5jLiUmDR1NPTo9tvv10FBQXas2fPoOMqKio0f/784O3k5OTRKG/MGaoffr9fa9asUWpqqnbt2qWvvvpKjzzyiCzLUnl5uQ0Vj033339/nxelxMREG6sZO9566y1VVFTo0Ucf1ezZs/XKK69o9erVevPNN5WZmWl3eWNSVlaWamtrg7fdbreN1YwtPp9POTk5WrJkicrKyvrt37Fjh2pra7V161ZNmzZN27dv16pVq/T2228rKSnJhorNNlQ/JGn+/PmqqKgI3o6NjQ1rLsLtEO6//35J0uuvv37ZcePHj1dGRsZolDSmDdWPDz/8UP/1X/+lDz74QBMnTpR04V+D69at09q1a3nBGiXjxo3j+WCD2tpaLV26VN/5znckST/5yU/04Ycfavfu3frhD39oc3Vjk9vt5rlgkwULFmjBggUD7rMsSzt37lRpaakWLVokSaqsrNTcuXO1b98+vgQqCi7Xj15erzcizxeWJUTIpk2bVFRUpKVLl2r37t0KBAJ2lzQm/eY3v1FWVlYw2EpSSUmJuru7dfjwYRsrG1uee+45FRUV6S/+4i+0fft2loWMgu7ubn300UcqKSnps33evHmqr6+3qSp8+umnKikp0cKFC7V27Vr94Q9/sLskSGpublZLS0uf54vX69WcOXN4vtjo4MGDKi4u1re+9S399Kc/VVtbW1jH4cxtBDzwwAMqLi5WfHy8Dhw4oMrKSp0+fVr33nuv3aWNOa2trUpPT++zLSUlRbGxsWptbbWpqrFl5cqVys3N1fjx49XQ0KC/+7u/U3Nzs7Zs2WJ3aUY7ffq0/H6/0tLS+mxPT09XS0uLTVWNbTNnzlRlZaWmTZumtra24GcG9u3bp9TUVLvLG9N6nxMDPV9OnDhhR0lj3o033qjbb79dmZmZam5u1tNPP6177rlHr7/+urxeb0jHGpPhtrq6Wtu2bbvsmD179ig/P39Yx7s4xH7961+XJD3zzDOE22GKdD9cLldI2zG0UHr0V3/1V8Ft1113ncaPH6/7779fDz30EP9DHwWX/p5blsXvvk0ufQu2oKBAt912m9544w2tWrXKpqpwsYGeL7DHHXfcEfx7dna28vLytHDhQn3wwQfBpSPDNSbD7d13393nQRzI5MmTwz7+rFmz1NnZOeBZRPQXyX6kp6frt7/9bZ9t7e3t6unp6fcvdAzfSHpUUFAgSfrss88It1GUmpoqt9vd7x2KtrY2XoccIjExUdnZ2Tp27JjdpYx5ves6W1tbNWHChOB2ni/OMWHCBGVmZob1fBmT4fbqq6/W1VdfHbXjHzlyRHFxcRo/fnzU5jBJJPtRUFCgf/iHf9CpU6eCL1h1dXXyer3Ky8uLyBxj0Uh69Lvf/U6S+FBNlHm9Xn3jG99QXV2dbrvttuD2/fv365ZbbrGxMvTq7u7Wxx9/rOuvv97uUsa8yZMnKyMjQ3V1dcrNzZV0oT+HDh3SQw89ZHN1kC4stfr888/7/ONjuMZkuA3FiRMn1N7erhMnTsjv9+vIkSOSpK997WsaN26c3n//fbW2tqqgoEDx8fH69a9/raqqKi1btizkNSIY2lD9KCkp0YwZM/Twww/r4YcfVnt7uyorK7Vs2TKulDAK6uvr9dvf/lZFRUVKSkpSQ0ODKioqtHDhQi5FNQpWrVqlhx9+WHl5eSosLNSrr76qzz//nE9+26SyslI333yzrr32Wn355Zfavn27Ojs7ddddd9ld2phw5swZffbZZ8Hbzc3NOnLkiFJSUpSZmamVK1eqpqZG06ZN09SpU1VTU6P4+HgtXrzYxqrNdbl+pKSkaNu2bVq0aJEyMjJ0/PhxVVVVKTU1VbfeemvIc7ksFphc1rp16/SLX/yi3/adO3eqqKhIv/zlL/XUU0/p008/lWVZmjJlir797W/r7rvvlsfDvx0ibah+SBcC8MaNG/Xv//7vwReqRx55hH9sjIKPPvpIGzdu1CeffKLu7m5lZmbqzjvv1Pe//30lJCTYXd6Y8PLLL+v555/XqVOnlJ2drR//+MeaM2eO3WWNSWvXrtWhQ4f01VdfKTU1VQUFBXrggQc0Y8YMu0sbE379619r5cqV/bbfdddd2rp1qyzL0rZt2/Tqq6+qvb1ds2bN0oYNG5SdnW1Dtea7XD8ee+wx3Xffffrd736njo4OZWRkqKioSA888ICuvfbakOci3AIAAMAYXOcWAAAAxiDcAgAAwBiEWwAAABiDcAsAAABjEG4BAABgDMItAAAAjEG4BQAAgDEItwAAADAG4RYAAADGINwCAADAGIRbAAAAGOP/AyHbZjPufhi1AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_stat(test_pos_scores_after_rank, test_neg_scores_after_rank)"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.g5.2xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 2.0.0 Python 3.10 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/pytorch-2.0.0-gpu-py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
